import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;

import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;


import de.tudarmstadt.ukp.wiki.parser.Paragraph;
import de.tudarmstadt.ukp.wiki.parser.ParsedPage;
import de.tudarmstadt.ukp.wikipedia.api.Category;
import de.tudarmstadt.ukp.wikipedia.api.DatabaseConfiguration;
import de.tudarmstadt.ukp.wikipedia.api.Page;
import de.tudarmstadt.ukp.wikipedia.api.WikiConstants;
import de.tudarmstadt.ukp.wikipedia.api.Wikipedia;
import de.tudarmstadt.ukp.wikipedia.api.exception.WikiApiException;
import de.tudarmstadt.ukp.wikipedia.api.exception.WikiPageNotFoundException;

/**
 * 
 * Wikipedia categories are used as a kind of semantic tag for pages.
 * They are organized in a thesaurus like structure.
 *
 */
public class Music_Category implements WikiConstants {
	
	static ArrayList<String> keywords = new ArrayList<String>();
	static ArrayList<String> stopwords = new ArrayList<String>();
	static ArrayList<String> paragraphTypes = new ArrayList<String>();

	public static void writeStringToFile(String file, String str){
		try {
			FileOutputStream fos = new FileOutputStream(file);
			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fos,"UTF-8"));
			bw.write(str);
			bw.flush();
			bw.close();
			fos.flush();
			fos.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	static void readKeywordsFromFile(String file){
		try {
			SAXBuilder builder = new SAXBuilder();
			Document d = builder.build(new FileInputStream(file));
			
			Element docroot = d.getRootElement();
			
			List<Element> keywordsList = docroot.getChildren("keyword");
			
			for(int i=0; i<keywordsList.size(); i++){
				Element keyword = keywordsList.get(i);
				keywords.add(keyword.getText().toLowerCase());
			}
				
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (JDOMException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	static void readStopwordsFromFile(String file){
		try {
			SAXBuilder builder = new SAXBuilder();
			Document d = builder.build(new FileInputStream(file));
			
			Element docroot = d.getRootElement();
			
			List<Element> stopwordsList = docroot.getChildren("stopword");
			
			for(int i=0; i<stopwordsList.size(); i++){
				Element keyword = stopwordsList.get(i);
				stopwords.add(keyword.getText().toLowerCase());
			}
				
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (JDOMException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	
    public static void main(String[] args) throws WikiApiException {
    	// 
    	readKeywordsFromFile("data\\Keywords.xml");
    	readStopwordsFromFile("data\\Stopwords.xml");
        
        // configure the database connection parameters
        DatabaseConfiguration dbConfig = new DatabaseConfiguration();
        dbConfig.setHost("ella.cl.uni-heidelberg.de");
        dbConfig.setDatabase("jwpl_en");
        dbConfig.setUser("jwpl_rw");
        dbConfig.setPassword("wikipedia");
        dbConfig.setLanguage(Language.english);

        // Create a new English wikipedia.
        Wikipedia wiki = new Wikipedia(dbConfig);

        // Get the category "Music"
        String title = "Music";
        Category cat;
        try {
            cat = wiki.getCategory(title);
        } catch (WikiPageNotFoundException e) {
            throw new WikiApiException("Category " + title + " does not exist");
        }

        StringBuilder sb = new StringBuilder();

        // the title of the category
        sb.append("Title : " + cat.getTitle() + LF);
        sb.append(LF);


        // the number of pages that are categorized under this category
        sb.append("# pages : " + cat.getPages().size() + LF);

        Integer numberOfGoodParagraphs = 0;
        
        for (Page page : cat.getPages()) {
        	sb.append("**********************\n");
            sb.append("***  " + page.getTitle() + LF);
            sb.append("**********************\n");
            
            ParsedPage ppage = page.getParsedPage();
            List<Paragraph> listParagraphs = ppage.getParagraphs();
            
            
            for(int k=0; k<listParagraphs.size(); k++){            	
            	if(!paragraphTypes.contains(listParagraphs.get(k).getType().toString())){
            		paragraphTypes.add(listParagraphs.get(k).getType().toString());
            	}
            	
            	Paragraph cParagraph = listParagraphs.get(k);
            	String cParagraphText = cParagraph.getText().toLowerCase();
            	Pattern cPattern = Pattern.compile("[</>*#-,.!?:\"\f\t\\v\\s]+");
            	Dictionary cDictionary = new Dictionary();
            	cDictionary.parseString(cParagraphText, cPattern);
            	Double cRatio = (double) cDictionary.getKeywordsCount(keywords) / cDictionary.getWordsCount();
            	
            	//System.out.println("Ratio = "+cDictionary.getKeywordsCount(keywords)+"/"+cDictionary.getWordsCount() + " for "+cDictionary.toString());
            	
            	if(cRatio > 0.0){
            		numberOfGoodParagraphs++;
            	}
            	
            	if(cRatio > 0.05){
            		
            		sb.append("- - - - - - - - \n");
            		sb.append("Paragraph :\n" );
            		sb.append(cParagraph.getText() + "\n");
            		sb.append(LF);
            		sb.append("Length of paragraph:" + cDictionary.getWordsCount() + LF);
            		sb.append("known Keywords:" + cDictionary.getKeywordsCount(keywords) + LF);
            		sb.append(LF);
            		
            		cDictionary.substractStopwordsList(stopwords);
            		cDictionary.substractKnownKeywordsList(keywords);
            		
            		sb.append("Dictionary :\n");
            		sb.append(cDictionary.toString() + "\n");
            		sb.append(LF);
            	}
            }
        }
        
        writeStringToFile("text.out", sb.toString());
        System.out.println("# of good paragraphs = "+numberOfGoodParagraphs);
    }
}